#Read and mean impute the training data
setwd("~/accept_decline")

mean_impute = function(x)
{
	mu = mean(na.omit(x))
	x[is.na(x)] = mu
	x
}

df = readRDS("train.rds")
df = data.frame(lapply(df, mean_impute))
df = data.frame(df, loss_ind=df$loss>0)

#Compute the Tantrev variable
m = glm(formula = loss_ind ~ f274 + f528, family = binomial(), data = df)
pred_tantrev = predict(m, type="response")
a = cut(pred_tantrev, quantile(pred_tantrev, seq(0, 1, .01), include.lowest=T))
plot(tapply(df$loss_ind, a, mean))
sum(df$loss_ind[pred_tantrev>.09])/sum(df$loss_ind)

quantilize = function(x)
{
  qntls = unique(quantile(x, seq(0, 1, .05)))
  print(length(qntls))
  if(length(qntls)>1)cut(x, qntls, 1:(length(qntls) - 1), include.lowest=T)
  else as.factor(rep(1, length(x)))
}

df2 = df[pred_tantrev > .09,]
pred_tantrev2 = pred_tantrev[pred_tantrev > .09]
a = cut(pred_tantrev2, quantile(pred_tantrev2, seq(0, 1, .2), include.lowest=T))
df_quantile = lapply(df2[,2:(ncol(df2) - 2)], quantilize)
df_quantile = data.frame(df_quantile, tantrev = a, loss_ind=df2$loss_ind, loss=df2$loss)

#What if I just do quantile regression on the loss here?
#m_fine = glm(loss_ind ~ tantrev + f13 + f67 + f84 + f404 + f514 + f527 + f201 + f211 + f238,
#             data=df_quantile, family=binomial)

m_fine = glm(loss_ind~ tantrev+f2+f13+
			     f404+f468+f471+f543+f588+f597+
			     f611+f615+f670+f674+f681+f683+
				 f712+f715+f726+f727+f772 + tantrev*f2,
	data=df_quantile, family=binomial)
pred = predict(m_fine, type="response", newdata=df_quantile)
pred[is.na(pred)] = 0
plot(quantile(pred, seq(0, 1, .05)), type="l")
b=cut(pred, quantile(pred, seq(0, 1, .05), include.lowest=T))
points(tapply(df_quantile$loss_ind, b, mean))
tapply(df_quantile$loss, b, median)


##Now need to apply to test, which means test variables must be quantilized
quantilize2 = function(x,y)
{
	qntls = unique(quantile(x, seq(0, 1, .05)))
	print(length(qntls))
	if(length(qntls)>1)cut(y, qntls, 1:(length(qntls) - 1), include.lowest=T)
	else as.factor(rep(1, length(y)))
}

df_test = readRDS("test.rds")


tantrev+f2+f13+
			     f404+f468+f471+f543+f588+f597+
			     f611+f615+f670+f674+f681+f683+
				 f712+f715+f726+f727+f772,

df_test = data.frame(id=df_test$id, lapply(
	df_test[ ,c("f2","f13",
			     "f404","f468","f471","f543","f588","f597",
			     "f611","f615","f670","f674","f681","f683",
				 "f712","f715","f726","f727","f772", "f274", "f528")],	mean_impute))
pred_test_tantrev = predict(m, newdata=df_test, type="response")
df_test2 = df_test[pred_test_tantrev>.09,]
pred_test_tantrev2 = pred_test_tantrev[pred_test_tantrev>.09]
b = cut(pred_test_tantrev2, quantile(pred_tantrev2, seq(0, 1, .2), include.lowest=T))

df_test_quantile = data.frame(id=df_test2$id, 
f2=quantilize2(df2$f2, df_test2$f2),
f13=quantilize2(df2$f13, df_test2$f13),
f404=quantilize2(df2$f404, df_test2$f404),
f468=quantilize2(df2$f468, df_test2$f468),
f471=quantilize2(df2$f471, df_test2$f471),
f543=quantilize2(df2$f543, df_test2$f543),
f588=quantilize2(df2$f588, df_test2$f588),
f597=quantilize2(df2$f597, df_test2$f597),
f611=quantilize2(df2$f611, df_test2$f611),
f615=quantilize2(df2$f615, df_test2$f615),
f670=quantilize2(df2$f670, df_test2$f670),
f674=quantilize2(df2$f674, df_test2$f674),
f681=quantilize2(df2$f681, df_test2$f681),
f683=quantilize2(df2$f683, df_test2$f683),
f712=quantilize2(df2$f712, df_test2$f712),
f715=quantilize2(df2$f715, df_test2$f715),
f726=quantilize2(df2$f726, df_test2$f726),
f727=quantilize2(df2$f727, df_test2$f727),
f772=quantilize2(df2$f772, df_test2$f772),
tantrev = b)

pred_test = predict(m_fine, type="response", newdata=df_test_quantile)

test_loss = rep(0, nrow(df_test2))
(0.556,0.703]       (0.703,0.794]       (0.794,0.846]       (0.846,0.879] 
1                   2                   3                   4 
(0.879,0.904]       (0.904,0.923]       (0.923,0.937]       (0.937,0.949] 
4                   4                   4                   5 
(0.949,0.958]       (0.958,0.968]       (0.968,0.977]       (0.977,0.999] 
6                   5                   6                   6 

test_loss[pred_test>.556 & pred_test<=.703] = 1
test_loss[pred_test>.703 & pred_test<=.794] =2
test_loss[pred_test>.794 & pred_test<=.846] = 3
test_loss[pred_test>.846 & pred_test<=.937] = 4
test_loss[pred_test>.937 & pred_test<=.968] = 5
test_loss[pred_test>.968] = 6



df_ans = data.frame(id=df_test2$id, test_loss)
df_ans2 = merge(data.frame(id=df_test$id), df_ans, by="id", all.x=T)
names(df_ans2)=c("id","loss")
df_ans2$loss[is.na(df_ans2$loss)] = 0

write.csv(df_ans2,"submission.csv", row.names=F, quote=F)
